import json
import re

import pymysql
import scrapy
from scrapy import Request

# 如果有多个爬虫，可以在这里写多个class，注意爬虫名称

from spider.item_rank_hot_video import SpiderItem, ChannelItem, subChannelItem, CommentItem, hot_video_lItem, VideoItem


class MarketIndustryoverviewSpiderChannel(scrapy.Spider):
    name = 'rank_hot_videodetail4'
    # def get_pid(self):
    #     conn = pymysql.connect(
    #         host="localhost",
    #         user="root",
    #         password="123456",
    #         db="b_video_stat",
    #         port=3306
    #     )
    #     cur = conn.cursor()
    #     sql1 = """
    #         SELECT pid FROM rank_hot_subvideo2;
    #     """
    #     cur.execute(sql1)
    #     data = cur.fetchall()
    #     data = [d1[0] for d1 in data]
    #     cur.close()
    #     conn.close()
    #     return data



#[,,,,,,,,,,,,,,,,,,,,,,,]

    def start_requests(self):
        channel_id = 10
        #a = [,,,,,,,,,,,,
        subchannel_id = 321905


        yield channel_id,subchannel_id

        # 近期热门
        url1 = 'https://api.bilibili.com/x/web-interface/web/channel/multiple/list?channel_id={}&sort_type=hot&offset=&page_size=30'.format(
            subchannel_id)
        # 播放最多
        url2 = 'https://api.bilibili.com/x/web-interface/web/channel/multiple/list?channel_id={}&sort_type=view&offset=&page_size=30'.format(
            subchannel_id)
        # 近期投稿
        url3 = 'https://api.bilibili.com/x/web-interface/web/channel/multiple/list?channel_id={}&sort_type=new&offset=&page_size=30'.format(
            subchannel_id)

        for url in [url1]:
            yield Request(url, meta={'channel_id': channel_id, 'subchannel_id': subchannel_id})


    def parse(self, response, **kwargs):

        channel_id = response.meta['channel_id']
        subchannel_id = response.meta['subchannel_id']

        content = json.loads(response.text)
        data = content.get('data', {})
        offset = data.get('offset', 0)
        has_more = data.get('has_more', False)
        videolist = data.get('list', [])
        for d in videolist:
            item = VideoItem()
            item['cid'] = channel_id
            item['scid'] = subchannel_id
            item['id'] = d.get('id', -1)
            item['name'] = d.get('name', -1)
            item['cover'] = d.get('cover', -1)
            item['view_count'] = transform_value(d.get('view_count', -1))
            item['like_count'] = transform_value(d.get('like_count', -1))
            item['duration'] = d.get('duration', -1)
            item['author_name'] = d.get('author_name', -1)
            item['author_id'] = d.get('author_id', -1)
            item['bvid'] = d.get('bvid', -1)
            item['card_type'] = d.get('card_type', -1)
            item['sort1'] = d.get('sort', -1)
            item['filt'] = d.get('filt', -1)

            # 获取视频的更多内容
            url = 'https://www.bilibili.com/video/{}'.format(item['bvid'])
            yield Request(url, meta={'item': item}, callback=self.parse2)

        if has_more:
            url = re.sub(r'offset=(.*)&', 'offset={}&'.format(offset), response.url)
            yield Request(url, meta={'channel_id': channel_id, 'subchannel_id': subchannel_id})

    def parse2(self, response):
        item = response.meta['item']
        values = response.xpath('//div[@class="video-data"]//text()').getall()
        # print(values)  # ['1246播放\xa0·\xa0', '2弹幕', '2021-05-17 21:15:10']
        danmu = values[1] if len(values) > 2 else ''
        danmu = danmu.replace('弹幕', '')
        danmu_count = danmu if danmu else 0

        pubdate = values[2] if len(values) > 2 else ''

        coins = response.xpath('//span[@class="coin"]/@title').get()
        coins = coins if coins else ''
        coins = coins.replace('投硬币枚数', '')
        coins_count = coins if coins else 0

        collect = response.xpath('//span[@class="collect"]/@title').get()
        collect = collect if collect else ''
        collect = collect.replace('收藏人数', '')
        collect_count = collect if collect else 0

        share = response.xpath('//span[@class="collect"]/@share/text()').get()
        share_count = share if share else 0

        item['danmu_count'] = danmu_count
        item['pubdate'] = pubdate
        item['coins_count'] = coins_count
        item['collect_count'] = collect_count
        item['share_count'] = share_count
        print('my item', item)
        yield item

def transform_value(s:str):
    s = str(s)
    if s.isdigit():
        return int(s)

    elif s.endswith('万'):
        s = s.replace('万', '')
        s = float(s)
        s *= 10000
        s = int(s)
        return s

    elif s.endswith('亿'):
        s = s.replace('亿', '')
        s = float(s)
        s *= 100000000
        s = int(s)
        return s

    else:
        return -1





